#include "wtlpch.h"

#include "CURLHelper.h"

#define _CURL_DEF_DOWNLOAD_BUFFER	1*1024*1024						//1M
#define _CURL_DEF_ONETIME_BUFFER	(CURL_MAX_WRITE_SIZE + 4*1024)	//Per Call transfer defined by curl is 16K, but i think a more 4k is safer

//Init static member
const LARGE_INTEGER				CCURLHelper::LINT_ZERO					= { 0, 0 };
CComAutoCriticalSection			CCURLHelper::sm_csTheadInit;
CComAutoCriticalSection			CCURLHelper::sm_csThreadWriteFile;
UINT							CCURLHelper::sm_uiDownBufferSize		= _CURL_DEF_DOWNLOAD_BUFFER;
UINT							CCURLHelper::sm_uiDownBufferThreshold	= _CURL_DEF_DOWNLOAD_BUFFER - _CURL_DEF_ONETIME_BUFFER;

size_t CCURLHelper::cb_write_stream_to_buffer(void* buffer, size_t size, size_t nmemb, void* userp)
{
	char* pstrTemp = *(char**)userp;
	char** ppStr = (char**)userp;

	memcpy(pstrTemp, buffer, size*nmemb);
	*ppStr = &pstrTemp[nmemb*size];
	
	return nmemb*size;
}

size_t CCURLHelper::cb_write_header_to_vector(void* buffer, size_t size, size_t nmemb, void* userp)
{

	((vector<string>*)userp)->push_back(string((char*)buffer));

	return nmemb*size;
}

size_t CCURLHelper::cb_http_header_router(void* buffer, size_t size, size_t nmemb, void* userp)
{
	TASK_DESCRIPTOR* pTask = (TASK_DESCRIPTOR*)userp;
	HEADER_HTTP_INFO* hti = pTask->pHttpInfo;

	//Hold the char in a string, and prepare a holding string
	char* lpstrHeaderLine = new char[size*nmemb + 1];
	::memset(lpstrHeaderLine, 0, size*nmemb + 1);
	::memcpy(lpstrHeaderLine, buffer, size*nmemb);
	string strHeaderLine(lpstrHeaderLine);

	//Find the content length, we reuse the heap buffer, it's safe here
	if(strHeaderLine.find(CHttpStreamHandler::RESP_HEADER[CHttpStreamHandler::CONTENTLENGTH]) != string::npos)
	{
		short sCurPos = 0;
		memset(lpstrHeaderLine, 0, strHeaderLine.length() + 1);

		for(const char* pcPos = strHeaderLine.c_str(); *pcPos != '\0'; pcPos++)
		{
			if(*pcPos >= '0' && *pcPos <= '9')
			{
				lpstrHeaderLine[sCurPos] = *pcPos;
				sCurPos++;
			}
		}

		//Make the buffer last zero for _atoi64 use
		lpstrHeaderLine[sCurPos] = '\0';

		hti->liContentLength.QuadPart = _atoi64(lpstrHeaderLine);

		//If this is a direct http download
		if(hti->liContentLength.QuadPart > 0)
		{
			hti->isDirectHTTP = true;
		}
	}

	//Find the redirect 
	if(strHeaderLine.find(CHttpStreamHandler::RESP_HEADER[CHttpStreamHandler::LOCATION]) != string::npos)
	{
		string::size_type szPos;

		//This is a redirection, extract the link, we handle ftp redirection our self, and put http redirection to curl
		if((szPos = strHeaderLine.find("ftp://")) != string::npos) 
		{
			//Set the rediction flag
			hti->isRedirectFTP = true;

			const char* pcPos = strHeaderLine.c_str();
			pcPos = &pcPos[szPos];
			short sCurPos = 0;

			for(; *pcPos != '\r' && *pcPos != '\n' && *pcPos != '\0'; pcPos++)
			{
				lpstrHeaderLine[sCurPos] = *pcPos;
				sCurPos++;				
			}
			
			//Close the char string
			lpstrHeaderLine[sCurPos] = '\0';

			hti->strLocation = lpstrHeaderLine;
		}
		else					//The redirection is http path or else etc. mms, rtsp, but now this program can't handle it
		{
			//Set the rediction flag
			//We let curl to handle it, so we need not to do anymore
			hti->isRedirectHTTP = true;
		}
	}

	if(hti->isRedirectFTP)		//If a ftp redirection response
	{
		//We handle ftp link our self, so stop automatic redirection here with a wrong return buffer size
		//CAUTION, SetOptionDisableFollowLocation won't work for a started transfer
		return -1;
	}

	if(hti->isRedirectHTTP)								//A http redirection, we handle it to curl
	{
		hti->ResetHttpStatus();
	}

	if(hti->isDirectHTTP && !hti->isHeaderProcessed)		//Now only remains a http download, so we perform the http download
	{
		LARGE_INTEGER liTempNewPos;

		//Make sure this block only excuted once
		hti->isHeaderProcessed = true;

		//We set the proper file virtual size for later data transfer
		::SetFilePointerEx(hti->hPreCreatedFile, hti->liContentLength, &liTempNewPos, FILE_BEGIN);
		::SetEndOfFile(hti->hPreCreatedFile);

		DOWN_FILE_DESCRIPTOR* pdf = new DOWN_FILE_DESCRIPTOR;
		
		pdf->bIsFTPLink = false;
		pdf->bIsPartDown = true;
		pdf->hFileHandle = hti->hPreCreatedFile;
		pdf->uiTotalFileSize = hti->liContentLength.LowPart;

		//For the other thread to start with this url, caution that there may be a redirection, 
		//the downlink url may diffrent with the request one, so we only take the current working url
		char* pcCurrentURL = new char[_CURL_MAX_URL_LENGTH];
		memset(pcCurrentURL, 0, _CURL_MAX_URL_LENGTH);

		//We only take a valid returned url
		CURLcode ccStatus = ::curl_easy_getinfo(*(hti->pCurlHelper), CURLINFO_EFFECTIVE_URL, &pcCurrentURL);
		if(ccStatus == CURLE_OK)
		{
			pdf->strFileLink = pcCurrentURL;
			pdf->strReferer = hti->strReferer;
		}
		else		//If error happpend, we took the requested url, though means useless
		{
			pdf->strFileLink = hti->strReqURL;
		}

		//Make task struct for every thread
		DivideVirtualFileIntoParts(hti->sUserSetThreads, pdf);

		//try to start all thread, this thread is also a downloading thread
		//We called it as main download thread, the thead also take over a thread vector 0, 
		//We assume index of the vector as main thread
 		hti->pCurlHelper->SetOptionWriteUserData(&pdf->vecTaskParts[0]);
		pdf->vecTaskParts[0].dwThreadID = ::GetCurrentThreadId();

		//Set the main thread data and function
		PART_DOWNLOAD* pPartDownInfo = new PART_DOWNLOAD(hti->hPreCreatedFile, pdf->vecTaskParts[0].uiPartStartPos, pdf->vecTaskParts[0].uiPartEndPos, pdf->vecTaskParts[0].sPartID);

		hti->pCurlHelper->SetOptionWriteFunction(cb_write_stream_part_file);
		hti->pCurlHelper->SetOptionWriteUserData(pPartDownInfo);

		ATLASSERT(pPartDownInfo != NULL);
		pdf->vecpPartDownInfo.push_back(pPartDownInfo);

		hti->pDownFileDesc = pdf;

		//And start the other thread, though we still read header, but we got enough information
		//the other thread won't check header anymore
		for(USHORT si = 1; si < pdf->vecTaskParts.size(); si++)
		{
			::_beginthread(CCURLHelper::HttpPartDownloadThread, NULL, pdf);
		}
	}

	//free the allocated buffer
	delete lpstrHeaderLine;

	return nmemb*size;
}

//Callback for get the first thread to get ftp header information
size_t CCURLHelper::cb_getinfo_ftp_header(void* buffer, size_t size, size_t nmemb, void* userp)
{
	//Extract ftp download info from the param
	TASK_DESCRIPTOR* pTask = (TASK_DESCRIPTOR*)userp;
	HEADER_FTP_INFO* ftpInfo = pTask->pFTPInfo;

	//Hold the header string buffer in a heap, and prepare a holding string
	//Cause the header string may not be null terminated, so we have to make it a null term
	char* lpstrHeaderLine = new char[size*nmemb + 1];
	::memset(lpstrHeaderLine, 0, size*nmemb + 1);
	::memcpy(lpstrHeaderLine, buffer, size*nmemb);
	string strHeaderLine(lpstrHeaderLine);

	//Find the download length, the resp code must be found in the beginning, or it may be other info
	if(strHeaderLine.find(CHttpStreamHandler::RESP_HEADER[CHttpStreamHandler::FILESTATUS]) == 0)
	{
		//The file size is after the file status resp
		ftpInfo->liFileSize.QuadPart = ::_atoi64(strHeaderLine.substr(CHttpStreamHandler::RESP_HEADER[CHttpStreamHandler::FILESTATUS].length()).c_str());
		ftpInfo->isFileReady = true;
	}

	//Find if we encounter a wrong login error
	if(strHeaderLine.find(CHttpStreamHandler::RESP_HEADER[CHttpStreamHandler::ERRORLOGIN]) == 0)
	{
		ftpInfo->isError = true;
		ftpInfo->sErrorType = HEADER_FTP_INFO::E_FTP_LOGIN;
	}

	//Find if we encounter file not exist error
	if(strHeaderLine.find(CHttpStreamHandler::RESP_HEADER[CHttpStreamHandler::FILENOEXIST]) == 0)
	{
		ftpInfo->isError = true;
		ftpInfo->sErrorType = HEADER_FTP_INFO::E_FTP_NO_EXIST;
	}

	//Find if the ftp server surport resume
	//With this surport, we should test REST Command in our ftp comamand line
	if(strHeaderLine.find(CHttpStreamHandler::RESP_HEADER[CHttpStreamHandler::MOREINFO]) == 0)
	{
		//With more details, we should check if server response these keywords to surport resume
		if(strHeaderLine.find("STOR") != string::npos || strHeaderLine.find("RETR") != string::npos)
		{
			ftpInfo->isResume = true;
		}
	}

	//The first thread is Succesfully connected, and ready to receive data, so we can start other thread now
	if(strHeaderLine.find(CHttpStreamHandler::RESP_HEADER[CHttpStreamHandler::READYTRANSFER]) == 0)
	{
		//Here means that the first thread is began, so we can start the other threads
		//Cause ftp cons are slow, sometimes this will bring less error from the server
		ATLASSERT(ftpInfo->pDownFileDesc != NULL);		//The Down file structrue must be filled

		for(USHORT si = 1; si < ftpInfo->pDownFileDesc->vecTaskParts.size(); si++)
		{
			::_beginthread(CCURLHelper::FtpPartDownloadThread, NULL, pTask);
		}		
	}

	//We got the filesize, so we logined and the file do exist, so we can set the write function
	if(ftpInfo->isFileReady)
	{
		LARGE_INTEGER liTempNewPos;

		//Make sure this block only excuted once
		ftpInfo->isFileReady = false;

		//We set the proper file virtual size for later data transfer
		::SetFilePointerEx(ftpInfo->hPreCreatedFile, ftpInfo->liFileSize, &liTempNewPos, FILE_BEGIN);
		::SetEndOfFile(ftpInfo->hPreCreatedFile);

		//Prepare the download file descriptor for other thread
		//This pointer will be the passed back to main thread, contained all thread info
		DOWN_FILE_DESCRIPTOR* pdownfileDescription = new DOWN_FILE_DESCRIPTOR;

		pdownfileDescription->bIsFTPLink = true;
		pdownfileDescription->bIsPartDown = true;
		pdownfileDescription->hFileHandle = ftpInfo->hPreCreatedFile;
		pdownfileDescription->uiTotalFileSize = ftpInfo->liFileSize.LowPart;
		pdownfileDescription->strFileLink = ftpInfo->strLocation;

		//Set the pointer of ftp header, though we can get the other thread's status
		ftpInfo->pDownFileDesc = pdownfileDescription;

		//if it resumeable, we try multithread download
		if(ftpInfo->isResume)
		{
			//Make task struct for every thread, Fill the task vector for pdf
			DivideVirtualFileIntoParts(ftpInfo->sUserSetThreads, pdownfileDescription);			
		}
		else
		{
			//If resume not surport, we ignore user's thread settings, just use single thread
			DivideVirtualFileIntoParts(1, pdownfileDescription);
		}

		//The header detection thread is the main thread
		//It also a downloading thread, we always set it to take the first part down entry
		//It's safe without CS, because all other thread are yet not started
		pdownfileDescription->vecTaskParts[0].dwThreadID = ::GetCurrentThreadId();

		//Set the main thread part down structrue
		PART_DOWNLOAD* pPartDownInfo = new PART_DOWNLOAD(ftpInfo->hPreCreatedFile, pdownfileDescription->vecTaskParts[0].uiPartStartPos, pdownfileDescription->vecTaskParts[0].uiPartEndPos, pdownfileDescription->vecTaskParts[0].sPartID);

		ftpInfo->pCurlHelper->SetOptionWriteFunction(cb_write_stream_part_file);
		ftpInfo->pCurlHelper->SetOptionWriteUserData(pPartDownInfo);

		ATLASSERT(pPartDownInfo != NULL);
		pdownfileDescription->vecpPartDownInfo.push_back(pPartDownInfo);
	}

	//Free the buffer we allocated for hold header string
	delete lpstrHeaderLine;

	return nmemb*size;
}


size_t CCURLHelper::cb_write_stream_to_file(void* buffer, size_t size, size_t nmemb, void* userp)
{
	DWORD dwWritten;
	BOOL bResult;

	BUFFER_DOWNFILE* bfd = (BUFFER_DOWNFILE*)userp;

	//We Don't know the file size, so do the incresing mode
	if(bfd->uiFileReportSize == 0)
	{	
		LARGE_INTEGER liNewFilePos;

		//Create a 2 times virtual file
		bResult = ::SetFilePointerEx(bfd->hFile, bfd->lintVirtualSteppingSize, &liNewFilePos, FILE_BEGIN);
		bResult = ::SetEndOfFile(bfd->hFile);

		//Reset to the beginning
		bResult = ::SetFilePointerEx(bfd->hFile, LINT_ZERO, &liNewFilePos, FILE_BEGIN);
		bfd->uiFileVirtualSize += bfd->lintVirtualSteppingSize.LowPart;

		bfd->uiFileReportSize = 1;
	}

	//Copy to the buffer
	memcpy(&bfd->pcBuffer[bfd->uiCurrentBufferedSize], buffer, size*nmemb);
	bfd->uiCurrentBufferedSize += size*nmemb;

	//Buffer nearly Fulled, write file
	if(bfd->uiCurrentBufferedSize > bfd->uiBufferThreshold)
	{
		//if this written cause exceeded, resize file
		if((bfd->uiFileWrittenSize + bfd->uiCurrentBufferedSize) > bfd->uiFileVirtualSize)
		{
			LARGE_INTEGER liPrevFilePos;
			LARGE_INTEGER liNewFilePos;

			//Save Curren file pos
			bResult = ::SetFilePointerEx(bfd->hFile, LINT_ZERO, &liPrevFilePos, FILE_CURRENT);
			//Enlarge File
			bResult = ::SetFilePointerEx(bfd->hFile, bfd->lintVirtualSteppingSize, &liNewFilePos, FILE_END);
			bfd->uiFileVirtualSize += bfd->lintVirtualSteppingSize.LowPart;
			bResult = ::SetEndOfFile(bfd->hFile);

			//Rewind to prev pos
			bResult = ::SetFilePointerEx(bfd->hFile, liPrevFilePos, &liNewFilePos, FILE_BEGIN);
		}

		//Write buffer to file, and reset the buffersize
		::WriteFile(bfd->hFile, bfd->pcBuffer, bfd->uiCurrentBufferedSize, &dwWritten, NULL);
		bfd->uiFileWrittenSize += dwWritten;
		bfd->uiCurrentBufferedSize = 0;
	}

	return size*nmemb;
}

size_t CCURLHelper::cb_write_stream_part_file(void* buffer, size_t size, size_t nmemb, void* userp)
{
	PART_DOWNLOAD* ptd = (PART_DOWNLOAD*)userp;

	//The file must created and allocated
	ATLASSERT(ptd->hEntireFileHandle != NULL);

	//Copy to the buffer
	memcpy(&ptd->pcBuffer[ptd->uiCurrentBufferedSize], buffer, size*nmemb);
	ptd->uiCurrentBufferedSize += size*nmemb;

	//Buffer nearly Fulled, write to file part
	//Or the buffer is even bigger or same than the part left size, we also need to write that to disk now
	if((ptd->uiCurrentBufferedSize > ptd->uiBufferThreshold) || (ptd->uiCurrentBufferedSize >= (ptd->uiFileVirtualPartSize - ptd->uiFileWrittenPartSize)))
	{
		DWORD dwWritten;
		BOOL bResult;

		LARGE_INTEGER liTempFilePos;
		LARGE_INTEGER liCurrentPos;

		//if this written cause exceeded the part border, reduce written size
		//the = is added for a finish check, until this is the last part, totally not happend
		//put the = judgement here to prevent too much compare
		if((ptd->uiFileWrittenPartSize + ptd->uiCurrentBufferedSize) >= ptd->uiFileVirtualPartSize)
		{
			ptd->bIsFinished = true;
			ptd->uiCurrentBufferedSize -= (ptd->uiFileWrittenPartSize + ptd->uiCurrentBufferedSize - ptd->uiFileVirtualPartSize);
		}

		//Write buffer to file, and reset the buffersize
		//Cause this will be called very offen, so we use the CS directly
		sm_csThreadWriteFile.Lock();
		
		//We do't download the last pos byte, cause it is downloaded by next part at begin
		//and the end position is file endpos + 1, so we need not to download that byte too
		liCurrentPos.QuadPart = ptd->lintPartStartPos.QuadPart + ptd->uiFileWrittenPartSize;
		bResult = ::SetFilePointerEx(ptd->hEntireFileHandle, liCurrentPos, &liTempFilePos, FILE_BEGIN);		
		::WriteFile(ptd->hEntireFileHandle, ptd->pcBuffer, ptd->uiCurrentBufferedSize, &dwWritten, NULL);

		sm_csThreadWriteFile.Unlock();

		ptd->uiFileWrittenPartSize += dwWritten;
		ptd->uiCurrentBufferedSize = 0;
	}

	//Although we should return the whole size for curl to work properly
	//But if we finished this part task, we break the transfer
	if(ptd->bIsFinished) return -1;
	return size*nmemb;
}


//Thread wrapper
void __cdecl CCURLHelper::HttpPartDownloadThread(void* pParam)
{
	ATLASSERT(pParam != NULL);

	//Get the downfile description from the param
	CCURLHelper::DOWN_FILE_DESCRIPTOR* pFileDescript = (CCURLHelper::DOWN_FILE_DESCRIPTOR*)pParam;

	//Thread Curl Handle & Part down structrue
	CCURLHelper cuThreadCurlHandle;
	PART_DOWNLOAD* pPartThreadDown = NULL;

	CComCritSecLock<CComAutoCriticalSection> csLock(sm_csTheadInit, false);
	HRESULT hr = csLock.Lock();
	//if the cs is not locked, we will fail and cause assert
	if (FAILED(hr))
	{
		ATLASSERT(0);
		return;
	}

	//Init Thead Curl handle with part info
	for(vector<PART_TASK_DESCRIPTOR>::iterator it = pFileDescript->vecTaskParts.begin(); it != pFileDescript->vecTaskParts.end(); ++it)
	{
		if((*it).dwThreadID == NULL)
		{
			(*it).dwThreadID = ::GetCurrentThreadId();
			
			//File the partdown structrue
			pPartThreadDown = new PART_DOWNLOAD(pFileDescript->hFileHandle, (*it).uiPartStartPos, (*it).uiPartEndPos, (*it).sPartID);
			pFileDescript->vecpPartDownInfo.push_back(pPartThreadDown);
			break;
		}
	}
	
	csLock.Unlock();		//Very Neccessary for a thread function

	//We should find a place for our thread to down
	//And prevent for wrong thread calling
	ATLASSERT(pPartThreadDown != NULL);
	ATLASSERT(!pFileDescript->bIsFTPLink);

	//Set Cookie Hoder and UserAgent, we kept saving the cookies
	cuThreadCurlHandle.SetOptionReqUserAgent(_ConfigApp.AppSettings.strJSUserAgent.c_str());
	cuThreadCurlHandle.SetOptionReqCookieFile(_ConfigApp.JSSettings.strJSCookieFilePath.c_str());
	cuThreadCurlHandle.SetOptionCookieJar(_ConfigApp.JSSettings.strJSCookieFilePath.c_str());

	//Set Connection timeout
	cuThreadCurlHandle.SetOptionConnectionTimeOut(_ConfigApp.AppSettings.lJSConnectionTimeOut);

	//Set Download link url & Referer
	cuThreadCurlHandle.SetOptionURL(pFileDescript->strFileLink.c_str());
	cuThreadCurlHandle.SetOptionReqReferer(pFileDescript->strReferer.c_str());

	//Set resume info, both for ftp and http
	cuThreadCurlHandle.SetOptionResumeFrom(pPartThreadDown->lintPartStartPos.LowPart);

	//Set down function and data
	cuThreadCurlHandle.SetOptionWriteFunction(cb_write_stream_part_file);
	cuThreadCurlHandle.SetOptionWriteUserData(pPartThreadDown);

	//do the transfer
	CURLcode cc;
	cc = cuThreadCurlHandle.DoEasyPerform();

	//If an error happend
	if(!pPartThreadDown->bIsFinished)
	{
		LONG lErrorCode;
		cc =  ::curl_easy_getinfo(cuThreadCurlHandle, CURLINFO_RESPONSE_CODE, &lErrorCode);
	}
	else
	{
		//flush any unwritten buffer to part file
		pPartThreadDown->FlushingBufferToFile();
	}
}

//Thread wrapper for start http download
void __cdecl CCURLHelper::StartHttpPartDownloadThread(void* pParam)
{
	ATLASSERT(pParam != NULL);

	//Get the Http Link download info from param
	CCURLHelper::TASK_DESCRIPTOR* pTaskInfo = (CCURLHelper::TASK_DESCRIPTOR*)pParam;

	CURLcode crc;
	CCURLHelper curlStart;

	//Create Http Header info and save it to task desc
	CCURLHelper::HEADER_HTTP_INFO* phttpInfo = new CCURLHelper::HEADER_HTTP_INFO;
	pTaskInfo->pHttpInfo = phttpInfo;
	
	//Set info data
	phttpInfo->pCurlHelper = &curlStart;
	phttpInfo->hPreCreatedFile = pTaskInfo->hFile;
	phttpInfo->strReqURL = pTaskInfo->strDownLoadLink;
	phttpInfo->strReferer = pTaskInfo->strReferer;
	phttpInfo->sUserSetThreads = pTaskInfo->sUserSetDownThread;

	//The Data Transfer Callback will be set in header router
	curlStart.SetOptionDisableAutoEncoding();				//No Encoding needed for download links
	curlStart.SetOptionEnableFollowLocation();				//We Enable http redirection
	curlStart.SetOptionEnableAutoFollowReferer();			//Referer is also important

	//Set Cookie Hoder and UserAgent, we kept saving the cookies
	curlStart.SetOptionReqUserAgent(_ConfigApp.AppSettings.strJSUserAgent.c_str());		//User Agent to pretent to be a browser
	curlStart.SetOptionReqCookieFile(_ConfigApp.JSSettings.strJSCookieFilePath.c_str());	//Cookie is important for server
	curlStart.SetOptionCookieJar(_ConfigApp.JSSettings.strJSCookieFilePath.c_str());

	//Set URL and referer
	curlStart.SetOptionURL(phttpInfo->strReqURL.c_str());	//Set Download link
	curlStart.SetOptionReqReferer(phttpInfo->strReferer.c_str());

	//Set Header Callback
	curlStart.SetOptionHeaderFunction(CCURLHelper::cb_http_header_router);		//Router header, may there is a ftp redirection
	curlStart.SetOptionHeaderUserData(pTaskInfo);			//The download info is used also by other thread

	//Set time out, do't set transfer timeout, or you will never finish a download
	curlStart.SetOptionConnectionTimeOut(_ConfigApp.AppSettings.lJSConnectionTimeOut);	//We do not want to wait forever

	//Start the header transfer
	curlStart.DoEasyPerform();

	//If the link is redirected to a ftp link, we start the ftp downloading
	if(phttpInfo->isRedirectFTP)
	{
		//Create a structrue for ftp download and add it to task
		CCURLHelper::HEADER_FTP_INFO* pftpInfo = new CCURLHelper::HEADER_FTP_INFO;

		//Copy http info to ftp
		pftpInfo->strLocation = phttpInfo->strLocation;
		pftpInfo->sUserSetThreads = phttpInfo->sUserSetThreads;

		pftpInfo->hPreCreatedFile = phttpInfo->hPreCreatedFile;
		pftpInfo->pCurlHelper = &curlStart;

		//Refresh Taskinfo
		delete pTaskInfo->pHttpInfo;
		pTaskInfo->bIsFTPDownLoad = true;
		pTaskInfo->pFTPInfo = pftpInfo;

		//We reuse the curl handle to perform a ftp download
		curlStart.SetOptionDisableFollowLocation();			//No redirection in ftp protocal
		curlStart.SetOptionFTPNoCWD();						//No CWD Command make transfer quiker
		curlStart.SetOptionFTPNoEPSV();						//Redure a step
		curlStart.SetOptionURL(pftpInfo->strLocation.c_str());		//Set the link url
		curlStart.SetOptionAdditionHeaderTestFTPResume();	//Test for resume capability

		curlStart.SetOptionHeaderFunction(CCURLHelper::cb_getinfo_ftp_header);
		curlStart.SetOptionHeaderUserData(pTaskInfo);

		//We start downloading
		::PostThreadMessageW(_Module.m_dwMainThreadID, WM_DOWNLOADING_STATUS, (WPARAM)pTaskInfo->pListItem, DOWN_FILE_DESCRIPTOR::ST_BEGINDOWNLOAD);
		crc = curlStart.DoEasyPerform();

		//Flush current ftp thread to buffer if no error
		if((!pftpInfo->isError) && (pftpInfo->pDownFileDesc->vecpPartDownInfo.size() != 0))
		{
			//We only flush buffer when the part is finishe
			if(pftpInfo->pDownFileDesc->vecpPartDownInfo[0]->bIsFinished)
			{
				pftpInfo->pDownFileDesc->vecpPartDownInfo[0]->FlushingBufferToFile();
			}

			//if finished, send finish message
			if(pftpInfo->pDownFileDesc->IsDownFinish())
			{
				pTaskInfo->bIsTaskFinished = true;
				::PostThreadMessageW(_Module.m_dwMainThreadID, WM_DOWNLOADING_STATUS, (WPARAM)pTaskInfo->pListItem, DOWN_FILE_DESCRIPTOR::ST_FINISH);
			}
		}

		//Exit Thread now
		return;
	}

	//Flush current http thread to buffer
	phttpInfo->pDownFileDesc->vecpPartDownInfo[0]->FlushingBufferToFile();
}

//Thread Wrapper
void __cdecl CCURLHelper::FtpPartDownloadThread(void* pParam)
{
	ATLASSERT(pParam != NULL);

	//Get the downfile description from the param
	TASK_DESCRIPTOR* pTask = (TASK_DESCRIPTOR*)pParam;
	CCURLHelper::DOWN_FILE_DESCRIPTOR* pFileDescript = pTask->pFTPInfo->pDownFileDesc;

	//Thread Curl Handle & Part down structrue
	CCURLHelper cuThreadCurlHandle;
	PART_DOWNLOAD* pPartThreadDown = NULL;

	CComCritSecLock<CComAutoCriticalSection> csLock(sm_csTheadInit, false);
	HRESULT hr = csLock.Lock();
	//if the cs is not locked, we will fail and cause assert
	if (FAILED(hr))
	{
		ATLASSERT(0);
		return;
	}

	//Init Thead Curl handle with part info
	for(vector<PART_TASK_DESCRIPTOR>::iterator it = pFileDescript->vecTaskParts.begin(); it != pFileDescript->vecTaskParts.end(); ++it)
	{
		if((*it).dwThreadID == NULL)
		{
			(*it).dwThreadID = ::GetCurrentThreadId();
			
			//File the partdown structrue
			pPartThreadDown = new PART_DOWNLOAD(pFileDescript->hFileHandle, (*it).uiPartStartPos, (*it).uiPartEndPos, (*it).sPartID);
			pFileDescript->vecpPartDownInfo.push_back(pPartThreadDown);
			break;
		}
	}
	
	csLock.Unlock();		//Very Neccessary for a thread function

	//We should find a place for our thread to down
	//And prevent for a wrong thread call
	ATLASSERT(pPartThreadDown != NULL);
	ATLASSERT(pFileDescript->bIsFTPLink);

	//Disable CWD and EPSV mode
	cuThreadCurlHandle.SetOptionFTPNoCWD();
	cuThreadCurlHandle.SetOptionFTPNoEPSV();

	//Set Download link url
	cuThreadCurlHandle.SetOptionURL(pFileDescript->strFileLink.c_str());

	//Set resume info, both for ftp and http
	cuThreadCurlHandle.SetOptionResumeFrom(pPartThreadDown->lintPartStartPos.LowPart);

	//Set down function and data
	cuThreadCurlHandle.SetOptionWriteFunction(cb_write_stream_part_file);
	cuThreadCurlHandle.SetOptionWriteUserData(pPartThreadDown);

	//do the transfer, and check for any error happend
	CURLcode cc;
	cc = cuThreadCurlHandle.DoEasyPerform();

	//If an error happend
	if(!pPartThreadDown->bIsFinished)
	{
		LONG lErrorCode;
		cc =  ::curl_easy_getinfo(cuThreadCurlHandle, CURLINFO_RESPONSE_CODE, &lErrorCode);
	}
	else		//We perform a successful download, now flush the file, and exit the thread
	{
		//flush any unwritten buffer to part file
		pPartThreadDown->FlushingBufferToFile();

		//if finished, send finish message
		if(pFileDescript->IsDownFinish())
		{
			pTask->bIsTaskFinished = true;
			::PostThreadMessageW(_Module.m_dwMainThreadID, WM_DOWNLOADING_STATUS, (WPARAM)pTask->pListItem, DOWN_FILE_DESCRIPTOR::ST_FINISH);
		}
	}
}

//Thread Wrapper for items sniffer
void __cdecl CCURLHelper::HttpItemsSnifferThread(void* pParam)
{
	ATLASSERT(pParam != NULL);

	//Get the downfile description from the param
	CCURLHelper::HTTP_ITEMS_SNIFFER* pHttpSniffer = (CCURLHelper::HTTP_ITEMS_SNIFFER*)pParam;

	//Prepared buffer
	CHAR* pcDecContentBuffer = new CHAR[BUFFER_DEC_HTTP_STREAM];		//Prepare for the gzip decode
	CHAR* pcContentBuffer = new CHAR[BUFFER_GZ_HTTP_STREAM];			//Buffer for what we received from curl
	WCHAR* pwUContentBuffer = new WCHAR[BUFFER_PLAIN_HTTP_STREAM];		//Buffer for the plain text html

	CURLcode crcRetCode;
	CCURLHelper crhItemsSniffer;

	vector<string> vecstrHeader;	//A vector to hold the response header
	CHAR* pbfPos = NULL;

	//Set Curl performance
	crhItemsSniffer.SetOptionDisableAutoEncoding();
	crhItemsSniffer.SetOptionEnableAutoFollowReferer();
	crhItemsSniffer.SetOptionEnableFollowLocation();
	crhItemsSniffer.SetOptionReqContentEncoding("gzip");		//TODO: Temperary set gzip here
	crhItemsSniffer.SetOptionReqUserAgent(_ConfigApp.AppSettings.strJSUserAgent.c_str());

	//Set time out
	crhItemsSniffer.SetOptionConnectionTimeOut(_ConfigApp.AppSettings.lJSConnectionTimeOut);
	crhItemsSniffer.SetOptionTransferTimeOut(_ConfigApp.AppSettings.lJSTransferTimeOut);

	//Set Cookie Hoder, we kept saving the cookies
	crhItemsSniffer.SetOptionReqCookieFile(_ConfigApp.JSSettings.strJSCookieFilePath.c_str());
	crhItemsSniffer.SetOptionCookieJar(_ConfigApp.JSSettings.strJSCookieFilePath.c_str());

	//Set Write Function
	crhItemsSniffer.SetOptionHeaderFunction(CCURLHelper::cb_write_header_to_vector);
	crhItemsSniffer.SetOptionHeaderUserData(&vecstrHeader);
	crhItemsSniffer.SetOptionWriteFunction(CCURLHelper::cb_write_stream_to_buffer);
	crhItemsSniffer.SetOptionWriteUserData(&pbfPos);
	
	//Now began to perform
	UINT uiUrlPos = 0;			//Set the status of url
	for(vector<string>::iterator it = (*(pHttpSniffer->pvecstrUrls)).begin(); it != (*(pHttpSniffer->pvecstrUrls)).end(); ++it, uiUrlPos++)
	{
		//Reset the http buffer
		memset(pcContentBuffer, 0, BUFFER_GZ_HTTP_STREAM);
		wmemset(pwUContentBuffer, 0, BUFFER_PLAIN_HTTP_STREAM);

		pbfPos = pcContentBuffer;
		vecstrHeader.clear();

		//Inform the dialog we start processing
		::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_STARTING);

		//Set the perform urls
		crhItemsSniffer.SetOptionURL((*it).c_str());

		//Do the perform
		crcRetCode = crhItemsSniffer.DoEasyPerform();

		if(crcRetCode == CURLE_OK)
		{
			//inform dialog http got ok, show processing mode
			::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_HTTPOK);

			//Confirm Gzip content, if is, decode gzip
			if(CHttpStreamHandler::IsGzipContent(&vecstrHeader))
			{
				//Inform that we got gzip stream
				::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_GETGZIP);

				::memset(pcDecContentBuffer, 0, BUFFER_DEC_HTTP_STREAM);

				//We use big buffer for item sniffer
				if(CHttpStreamHandler::DecodeGzipStream(pcContentBuffer, CHttpStreamHandler::GetHeaderContentLength(&vecstrHeader), pcDecContentBuffer, true) != S_OK)
				{
					//A decode error happend here
					::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_DECODESTREAMERROR);
					//We just ignore this one
					continue;
				}
				
				//Gzip decode succesffuly
				::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_DECODESTREAMOK);

				//Now convert the decode html to UNICODE
				::MultiByteToWideChar(CP_UTF8, 0, pcDecContentBuffer, -1, pwUContentBuffer, BUFFER_PLAIN_HTTP_STREAM);
			}
			else	//Not a gzip content, we got plain content directly, so we go directly unicode transfer
			{
				//Inform that we get a plain stream
				::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_GETPLAIN);

				::MultiByteToWideChar(CP_UTF8, 0, pcContentBuffer, -1, pwUContentBuffer, BUFFER_PLAIN_HTTP_STREAM);
			}

			//Inform that we began to search item
			::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_SNIFFERBEGIN);

			//Conver Linebreak to whitespace for later use
			CHttpStreamHandler::RemoveLineBreak(pwUContentBuffer);
			
			//Now search the items from the html
			int nFoundItemCount = 0;
			nFoundItemCount = CHttpStreamHandler::JSTwoLayerSearch((*pHttpSniffer->pmapJSItems)[*it], pwUContentBuffer, _ConfigApp.JSSettings.wstrJSHTMLOuterRegex.c_str(), _ConfigApp.JSSettings.wstrJSHTMLInnerRegex.c_str());
			nFoundItemCount = CHttpStreamHandler::JSTwoLayerSearch((*pHttpSniffer->pmapJSItems)[*it], pwUContentBuffer, _ConfigApp.JSSettings.wstrJSBBCodeOuterRegex.c_str(), _ConfigApp.JSSettings.wstrJSBBCodeInnerRegex.c_str());
			
			//Current link with no items, we inform the dialog
			if(nFoundItemCount == 0)
			{
				::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_SNIFFERZERO);
				continue;
			}

			//Got the SecIP for generate Download link, we donot care if we really found one cause a default one will replace while fail
			CHttpStreamHandler::JSGetSecIP(*it, *pHttpSniffer->pmapJSSecIp, pwUContentBuffer, _ConfigApp.JSSettings.wstrJSSecIPRegex.c_str());

			//We found some link successfully
			::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_SNIFFEROK);
		}
		else		//Opps, some error happend while connecting
		{
			//Wether we got a timeout 
			if(crcRetCode == CURLE_OPERATION_TIMEDOUT)
			{
				::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_TIMEOUT);
			}
			else
			{
				//Most of this case is CURLE_COULDNT_CONNECT
				::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_HTTPFAIL);
			}

			continue;
		}

		//Don't give the server too much load
		//We sniffer next url after Some second
		::Sleep(_ConfigApp.AppSettings.uiJSItemSnifferTimeWait);
	}

	//We finished all task, now over the job
	::PostThreadMessageW(pHttpSniffer->dwCallThreadID, WM_HTTP_ITEM_RESULT, (WPARAM)uiUrlPos, (LPARAM)HTTP_ITEMS_SNIFFER::ST_OVERJOB);

	//Don't forget to release buffer
	delete pcDecContentBuffer;
	delete pcContentBuffer;
	delete pwUContentBuffer;
}

//Thread Wrapper for Link sniffer, we use small buffer
void __cdecl CCURLHelper::PageLinkSnifferThread(void* pParam)
{
	ATLASSERT(pParam != NULL);

	//Get the downfile description from the param
	vector<CCURLHelper::PAGE_LINK_ITEM*>* plstLinkSniffer = (vector<CCURLHelper::PAGE_LINK_ITEM*>*)pParam;

	//Prepared buffer
	CHAR* pcDecContentBuffer = new CHAR[BUFFER_DEC_HTTP_STREAM_S];		//Prepare for the gzip decode
	CHAR* pcContentBuffer = new CHAR[BUFFER_GZ_HTTP_STREAM_S];			//Buffer for what we received from curl
	WCHAR* pwUContentBuffer = new WCHAR[BUFFER_PLAIN_HTTP_STREAM_S];		//Buffer for the plain text html

	CURLcode crcRetCode;
	CCURLHelper crhItemsSniffer;

	vector<string> vecstrHeader;	//A vector to hold the response header
	CHAR* pbfPos = NULL;

	//Set Curl performance
	crhItemsSniffer.SetOptionDisableAutoEncoding();
	crhItemsSniffer.SetOptionEnableAutoFollowReferer();
	crhItemsSniffer.SetOptionEnableFollowLocation();
	crhItemsSniffer.SetOptionReqContentEncoding("gzip");		//TODO: Temperary set gzip here
	crhItemsSniffer.SetOptionReqUserAgent(_ConfigApp.AppSettings.strJSUserAgent.c_str());

	//Set time out
	crhItemsSniffer.SetOptionConnectionTimeOut(_ConfigApp.AppSettings.lJSConnectionTimeOut);
	crhItemsSniffer.SetOptionTransferTimeOut(_ConfigApp.AppSettings.lJSTransferTimeOut);

	//Set Cookie Hoder, we kept saving the cookies
	crhItemsSniffer.SetOptionReqCookieFile(_ConfigApp.JSSettings.strJSCookieFilePath.c_str());
	crhItemsSniffer.SetOptionCookieJar(_ConfigApp.JSSettings.strJSCookieFilePath.c_str());

	//Create the COM Interface, cause of this is another thread, so we have to enter a STA
	::CoInitialize(NULL);
	IJsSiteHelper* piJSHelper;
	::CoCreateInstance(CLSID_JsSiteHelper, NULL, CLSCTX_INPROC_SERVER, IID_IJsSiteHelper, (void**)&piJSHelper);

	//Now began to perform
	CHAR pcCaptcha[10] = { 0 };
	bool bFillCaptcha = false;

	for(vector<CCURLHelper::PAGE_LINK_ITEM*>::iterator it = plstLinkSniffer->begin(); it != plstLinkSniffer->end(); bFillCaptcha ? it : ++it)
	{
		//Reset the http buffer
		memset(pcContentBuffer, 0, BUFFER_GZ_HTTP_STREAM_S);
		wmemset(pwUContentBuffer, 0, BUFFER_PLAIN_HTTP_STREAM_S);

		pbfPos = pcContentBuffer;
		vecstrHeader.clear();

		//Notify we began a item
		::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_STARTITEM);

		//Set Write Function
		crhItemsSniffer.SetOptionHeaderFunction(CCURLHelper::cb_write_header_to_vector);
		crhItemsSniffer.SetOptionHeaderUserData(&vecstrHeader);
		crhItemsSniffer.SetOptionWriteFunction(CCURLHelper::cb_write_stream_to_buffer);
		crhItemsSniffer.SetOptionWriteUserData(&pbfPos);

		//Set the referer for link page
		crhItemsSniffer.SetOptionReqReferer((*it)->strURLRefer.c_str());

		//Generate and Set the perform urls
		if(bFillCaptcha)
		{
			GeneratePageLink(piJSHelper, *(*it), pcCaptcha);
			bFillCaptcha = false;
		}
		else
		{
			GeneratePageLink(piJSHelper, *(*it));
		}

		crhItemsSniffer.SetOptionURL((*it)->strLinkPage.c_str());

		//Do the perform and notify the list
		::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_BEGIN_CONNECT);
		crcRetCode = crhItemsSniffer.DoEasyPerform();

		if(crcRetCode == CURLE_OK)
		{
			//Inform we connect succeffully
			::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_HTTP_OK);

			//Confirm Gzip content, if is, decode gzip
			if(CHttpStreamHandler::IsGzipContent(&vecstrHeader))
			{
				//Inform we got gzip content
				::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_HTTP_GZIP);

				::memset(pcDecContentBuffer, 0, BUFFER_DEC_HTTP_STREAM_S);

				//We use big buffer for item sniffer
				if(CHttpStreamHandler::DecodeGzipStream(pcContentBuffer, CHttpStreamHandler::GetHeaderContentLength(&vecstrHeader), pcDecContentBuffer, false) != S_OK)
				{
					//Inform the fail message
					::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_HTTP_GZDECFAIL);

					//We just ignore this one
					continue;
				}

				//We Decode Gzip succesffuly
				::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_HTTP_GZDECOK);
				
				//Now convert the decode html to UNICODE
				::MultiByteToWideChar(CP_UTF8, 0, pcDecContentBuffer, -1, pwUContentBuffer, BUFFER_PLAIN_HTTP_STREAM_S);
			}
			else	//Not a gzip content, we got plain content directly, so we go directly unicode transfer
			{
				//We got plain html
				::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_HTTP_PLAIN);

				::MultiByteToWideChar(CP_UTF8, 0, pcContentBuffer, -1, pwUContentBuffer, BUFFER_PLAIN_HTTP_STREAM_S);
			}

			//Conver Linebreak to whitespace for later use
			CHttpStreamHandler::RemoveLineBreak(pwUContentBuffer);

			//Check whether we should enter the captcha
			if(CHttpStreamHandler::IsCaptchaNeeded(pwUContentBuffer))
			{
				//A Captcha required
				::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_NEED_CAPTCHA);

				//Get the captcha Link
				string strCaptchaURL;
				CHttpStreamHandler::GetCaptchaLink(pwUContentBuffer, strCaptchaURL);

				//Save the captcha into a tempfile, we assume the captcha image won't use gzip here
				//Set Write Function, this time we only care about content, we write the stream into a temp file
				//Cause the file name will be used by OpenCV, so we assume it to be ansi version, and we assume the image is jpg type
				string strTempFileName;
				HANDLE hFile = CHttpStreamHandler::CreateTempFileWithExtA(".jpg", strTempFileName);

				BUFFER_DOWNFILE bfdFile(hFile);
				crhItemsSniffer.SetOptionURL(strCaptchaURL.c_str());
				crhItemsSniffer.SetOptionWriteFunction(CCURLHelper::cb_write_stream_to_file);
				crhItemsSniffer.SetOptionWriteUserData(&bfdFile);

				CURLcode crc = crhItemsSniffer.DoEasyPerform();
				bfdFile.FlushingBufferToFile();

				//Rec the captcha, we do't care wether success cause we will sent a default one
				::memset(pcCaptcha, 0, 10);
				piJSHelper->ReadCaptchaFromImage(strTempFileName.c_str(), pcCaptcha);
				(*it)->strCaptcha = pcCaptcha;

				//Now we got the captcha, so we require the url agan
				::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_REC_CAPTCHA);
				bFillCaptcha = true;

				//Delete Our tempfile
				::DeleteFileA(strTempFileName.c_str());

				//We Request this link again
				continue;
			}
			else		//Good, No Captcha Anymore
			{
				//Read Download link from the source, if fail, we inform the window
				if(!CHttpStreamHandler::GetDownLoadLinkFromJSPage(pwUContentBuffer, (*it)->strFileLink, (*it)->strDownLink))
				{
					//Opps, failed to get link
					::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_GETLINKFAIL);

					//We supress this item
					continue;
				}
				
				//Set the flag for success
				(*it)->bIsFoundLink = true;

				//Extract Filename on the server
				crhItemsSniffer.GetDecodedURLFileName((*it)->strFileLink.c_str(), (*it)->wstrServerFileName);

				//Encode the ftp url of http type, Generate the final link
				string strTempFtp = (*it)->strFileLink;
				crhItemsSniffer.EncodedPlainURL(strTempFtp);
				(*it)->strDownLink = (*it)->strDownLink + strTempFtp;

				//Notify of success
				::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_GETLINKOK);
			}

		}
		else		//Opps, some error happend while connecting
		{
			//Wether we got a timeout 
			if(crcRetCode == CURLE_OPERATION_TIMEDOUT)
			{
				//Inform for time out
				::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_TIMEOUT);
			}
			else
			{
				//Most of this case is CURLE_COULDNT_CONNECT
				::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_HTTPFAIL);
			}
			
			//We supress the failed one
			continue;
		}

		//Don't give the server too much load
		//We sniffer next url after Some second
		::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, (WPARAM)(*it)->pListItem, PAGE_LINK_ITEM::ST_WAITFORNEXT);
		::Sleep(_ConfigApp.AppSettings.uiJSItemSnifferTimeWait);
	}

	//Release Interface
	piJSHelper->Release();
	//Don't forget to leave STA
	::CoUninitialize();

	//Notify That we finished link get
	::PostThreadMessageW(_Module.m_dwMainThreadID, WM_TA_SNIFFER_LINK_STATUS, 0, PAGE_LINK_ITEM::ST_FINISH);

	//Don't forget to release buffer
	delete pcDecContentBuffer;
	delete pcContentBuffer;
	delete pwUContentBuffer;
}

//Create Pagelink
void CCURLHelper::GeneratePageLink(IJsSiteHelper* piJSHelper, PAGE_LINK_ITEM& itemDown, CHAR* pcCaptcha)
{
	//Prepare for link generation
	ATLASSERT(piJSHelper != NULL);
	
	//Generate Link page link, we assume the fileID is no longger than 36, and the link url is no longer than 512
	CHAR pstrFileID[36] = { 0 };
	CHAR pstrLinkURL[512] = { 0 };
	::WideCharToMultiByte(CP_ACP, WC_NO_BEST_FIT_CHARS, itemDown.wstrFileID.c_str(), -1, pstrFileID, 36, NULL, NULL);
	HRESULT hr = piJSHelper->GernerateLink(pstrFileID, itemDown.strSecIp.c_str(), pcCaptcha, pstrLinkURL);
	itemDown.strLinkPage = pstrLinkURL;
}

//Divide the file into serveral task, set the begin end pos, and part id
void CCURLHelper::DivideVirtualFileIntoParts(SHORT sParts, CCURLHelper::DOWN_FILE_DESCRIPTOR* pDownFileDescriptor)
{
	DOWN_FILE_DESCRIPTOR* pdfd = (DOWN_FILE_DESCRIPTOR*)pDownFileDescriptor;
	UINT uiDividePartSize = pdfd->uiTotalFileSize/sParts;

	ATLASSERT(uiDividePartSize > 0);

	//Divide the size of file, and push them in to vector
	for(SHORT si = 0; si < sParts; si++)
	{
		PART_TASK_DESCRIPTOR ptd;
		ptd.sPartID = si;
		ptd.uiPartStartPos = uiDividePartSize * si;
		ptd.uiPartEndPos = (si == (sParts - 1) ? pdfd->uiTotalFileSize : uiDividePartSize * (si + 1));

		pdfd->vecTaskParts.push_back(ptd);
	}
}

